{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright 2020 Google LLC.                                                                                                                                   \n",
    "#                                                                                                                                                              \n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");                                                                                              \n",
    "# you may not use this file except in compliance with the License.                                                                                             \n",
    "# You may obtain a copy of the License at                                                                                                                      \n",
    "#                                                                                                                                                              \n",
    "#     http://www.apache.org/licenses/LICENSE-2.0                                                                                                               \n",
    "#                                                                                                                                                              \n",
    "# Unless required by applicable law or agreed to in writing, software                                                                                          \n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,                                                                                            \n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.                                                                                     \n",
    "# See the License for the specific language governing permissions and                                                                                          \n",
    "# limitations under the License. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os,sys\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels.api as sm\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.summary.summary_iterator import summary_iterator\n",
    "from scipy import stats\n",
    "\n",
    "import seaborn as sns\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Helper functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract tf board data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extraction function\n",
    "def sum_log(path,config,model,runlog):\n",
    "#     try:\n",
    "    r={}\n",
    "    for e in summary_iterator(path):\n",
    "        for v in e.summary.value:\n",
    "            if v.tag not in r:\n",
    "                r[v.tag] = {'Model': model, 'config': config, 'Metric': v.tag, 'Values': [], 'Steps': []}\n",
    "            r[v.tag]['Values'].append(v.simple_value)\n",
    "            r[v.tag]['Steps'].append(e.step)\n",
    "    for k,v in r.items():\n",
    "        runlog = runlog.append(v, ignore_index=True)\n",
    "    \n",
    "    return runlog\n",
    "\n",
    "# load tf logs\n",
    "def log_tflogs_from_path(tfboardroot, trim_config=None, trim_model=None):\n",
    "    models = os.listdir(tfboardroot)\n",
    "\n",
    "    all_log = pd.DataFrame(columns=['Model', 'config', 'Metric', 'Values', 'Steps'])\n",
    "\n",
    "    modellist = set()\n",
    "    configlist = set()\n",
    "\n",
    "    for m in tqdm(models):\n",
    "        modelroot = os.path.join(tfboardroot,m)\n",
    "        modellist.add(m)\n",
    "        for config in os.listdir(modelroot):\n",
    "            configlist.add(config)\n",
    "            logpath = os.path.join(modelroot,config,'log')\n",
    "            for f in os.listdir(logpath):\n",
    "                if f.startswith('events'):\n",
    "                    logpath = os.path.join(logpath,f)\n",
    "                    break\n",
    "\n",
    "            all_log = sum_log(logpath,config,m,all_log)\n",
    "    if trim_config is not None:\n",
    "        all_log['config'] = all_log['config'].apply(trim_config)\n",
    "        configlist = set(all_log['config'])\n",
    "    if trim_model is not None:\n",
    "        all_log['Model'] = all_log['Model'].apply(trim_model)\n",
    "        modellist = set(all_log['Model'])\n",
    "    return all_log, modellist, configlist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Curve plot helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_plot_df(sub_human_stats, sub_log, metric, std_scale = 1.645/10):\n",
    "    steps = sub_log.iloc[0]['Steps']\n",
    "    sub_human_stats['Steps'] = [steps] * sub_human_stats.shape[0]\n",
    "    sub_human_stats = sub_human_stats.explode('Steps')\n",
    "    sub_log = sub_log[['Model','Values','Steps']]\n",
    "    \n",
    "    y = sub_log.iloc[0]['Values']\n",
    "    \n",
    "    sub_log = sub_log.set_index(['Model']).apply(pd.Series.explode).reset_index()\n",
    "    sub_log = sub_log.infer_objects()\n",
    "    value_mean = sub_log.groupby(['Model'])['Values'].apply(lambda x: x.ewm(halflife=10).mean())\n",
    "    value_std = sub_log.groupby(['Model'])['Values'].apply(lambda x: x.ewm(halflife=10).std()) \n",
    "    \n",
    "    plot_log = sub_log.copy()\n",
    "\n",
    "    plot_log['Evaluation']='OPE'\n",
    "    \n",
    "    sub_human_stats['Evaluation']='Truth'\n",
    "    new_plot_log = sub_human_stats.copy()\n",
    "    new_plot_log['Values'] = new_plot_log[metric+'-mean']\n",
    "    plot_log = plot_log.append(new_plot_log[['Model','Values','Steps','Evaluation']], ignore_index=True)\n",
    "    new_plot_log = sub_human_stats.copy()\n",
    "    if std_scale > 0:\n",
    "        new_plot_log['Values'] = new_plot_log[metric+'-mean'] + new_plot_log[metric+'-std']*std_scale #90% confidenen interval\n",
    "        plot_log = plot_log.append(new_plot_log[['Model','Values','Steps','Evaluation']], ignore_index=True)\n",
    "        new_plot_log = sub_human_stats.copy()\n",
    "        new_plot_log['Values'] = new_plot_log[metric+'-mean'] - new_plot_log[metric+'-std']*std_scale\n",
    "        plot_log = plot_log.append(new_plot_log[['Model','Values','Steps','Evaluation']], ignore_index=True)\n",
    "    \n",
    "    return plot_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate OPE helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gather_ope_logs_by_config(\n",
    "    all_log, \n",
    "    plot_metrics, \n",
    "    config, \n",
    "    last_n = 100,\n",
    "    trim_metric= lambda x: 'est_reward_dual_{}_normalized'.format(x),\n",
    "    final_n = None\n",
    "):\n",
    "\n",
    "    all_plotlog = None\n",
    "    for metric in plot_metrics:\n",
    "        plot_log = human_stats[['Model', metric+'-mean', metric+'-std']]\n",
    "        plot_log = plot_log.rename(columns={metric+'-mean': 'Human-mean', metric+'-std': 'Human-std'})\n",
    "        sub_log = all_log.loc[all_log['config']==config]\n",
    "        sub_log = sub_log.loc[sub_log['Metric']==trim_metric(metric)]\n",
    "        sub_log = sub_log[['Model','Values']]\n",
    "        if final_n is None:\n",
    "            sub_log['Values'] = sub_log['Values'].apply(lambda x: np.array(x[-last_n:]).mean())\n",
    "        else:\n",
    "            sub_log['Values'] = sub_log['Values'].apply(lambda x: np.array(x[-last_n:-final_n]).mean())\n",
    "        sub_log = sub_log.rename(columns={'Values':'OPE'})\n",
    "\n",
    "        plot_log = plot_log.set_index('Model')\n",
    "        sub_log = sub_log.set_index('Model')\n",
    "\n",
    "        plot_log = plot_log.join(sub_log)\n",
    "        plot_log['Metric'] = metric\n",
    "        if all_plotlog is None:\n",
    "            all_plotlog = plot_log\n",
    "        else:\n",
    "            all_plotlog = all_plotlog.append(plot_log)\n",
    "    all_plotlog['Error'] = (all_plotlog['OPE']-all_plotlog['Human-mean']).apply(lambda x: abs(x))\n",
    "\n",
    "    return all_plotlog"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AirDialogue Rule Based"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotroot=\"outputs/plotdir/air_ope/rule/\"\n",
    "os.makedirs(plotroot,exist_ok=True)\n",
    "tfboardroot = 'outputs/syn_air_ope/syn_ope_data_500'\n",
    "default_config = 'epoch_500'\n",
    "\n",
    "human_stats = {\n",
    "    'Model': ['L0', 'L1', 'L2', 'L3', 'L4', 'L5'], \n",
    "    'reward-mean': [0.4928, 0.5914, 0.6917, 0.7958, 0.8973, 1.0000], \n",
    "    'reward-std': [0.3292, 0.3589, 0.3574, 0.3261, 0.2525, 0.0001]\n",
    "}\n",
    "human_stats = pd.DataFrame(data=human_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "human_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_log, modellist, configlist = log_tflogs_from_path(tfboardroot, \n",
    "                                                      trim_config = lambda x: 'epoch_'+x.split('epoch_')[-1].split('_')[0],\n",
    "                                                      trim_model = lambda x: x.replace('tgt_', ''))\n",
    "\n",
    "print(modellist)\n",
    "print(configlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_log = all_log[all_log['config']==default_config]\n",
    "sub_log = sub_log[sub_log['Metric']=='est_reward_dual_normalized']\n",
    "\n",
    "plot_log = get_plot_df(human_stats, sub_log, 'reward', std_scale=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_log = plot_log.sort_values(by='Model')\n",
    "g = sns.lineplot(x='Steps', y='Values', hue='Model', style='Evaluation', data=plot_log)\n",
    "g.legend(loc='center left', bbox_to_anchor=(1.0, 0.5), ncol=1)\n",
    "g.set_position([0.15,0.15,0.6,0.8])\n",
    "g.figure.savefig(plotroot+'learning_curve.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Human vs OPE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_plotlog = gather_ope_logs_by_config(all_log, ['reward'], default_config, trim_metric=lambda x:'est_reward_dual_normalized')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_plotlog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def r2(x, y):\n",
    "    return stats.pearsonr(x, y)[0] ** 2\n",
    "all_plotlog = all_plotlog.rename(columns={'Human-mean': 'Reward', 'OPE': 'OPE'})\n",
    "g = sns.jointplot('Reward', 'OPE', kind=\"reg\", data=all_plotlog, stat_func=r2,\n",
    "             xlim=[0.4,1.1],ylim=[0.4,1.1])\n",
    "g.savefig(plotroot+'ope_vs_human.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AirDialogue Model-Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotroot=\"outputs/plotdir/air_ope/model/\"\n",
    "os.makedirs(plotroot,exist_ok=True)\n",
    "\n",
    "# load stats\n",
    "statspath = 'data/selfplay_opedata/orig/stats.csv'\n",
    "\n",
    "human_stats = pd.read_csv(statspath)\n",
    "human_stats = human_stats.rename(columns={'model_name': 'Model'}).set_index('Model')\n",
    "\n",
    "human_stats_auto = {\n",
    "    'Model': ['5K','10K','20K','30K','40K','50K','75K','100K','150K','200K','250K','full',\n",
    "              '5K_w','10K_w','20K_w','30K_w','40K_w','50K_w','75K_w','100K_w','150K_w','200K_w','250K_w','full_w'], \n",
    "    'ppl': [2.671, 2.368, 2.141, 2.053, 2.121, 1.919, 1.843, 2.084, 2.021, 2.076, 1.949, 1.954,\n",
    "            2.673, 2.369, 2.141, 2.053, 2.121, 1.919, 1.843, 2.084, 2.021, 2.076, 1.949, 1.954], \n",
    "    'BLEU': [12.53, 19.07, 23.72, 24.49, 19.64, 29.98, 31.70, 20.10, 21.95, 20.24, 26.04, 25.41,\n",
    "             12.53, 19.07, 23.71, 24.49, 19.64, 29.98, 31.70, 20.11, 21.95, 20.23, 26.04, 25.41], \n",
    "}\n",
    "human_stats_auto = pd.DataFrame(data=human_stats_auto).set_index('Model')\n",
    "human_stats = pd.concat([human_stats_auto, human_stats], axis=1, join='inner').reset_index()\n",
    "\n",
    "metriclist = ['flight_score', 'reward', 'status_score']\n",
    "print(metriclist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "human_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfboardroot = \"outputs/selfplay_air_ope_all\"\n",
    "default_config = 'roberta-base_fix_false_share_true_freeze_true_epoch_300_invsqrt_adam_lr_2e-4_C_1_Q_2_L_10x100_BERT_1_warmup_30_mom_0.5_MAXNORM_1_WD_1e-4_BS_20x1_Linit_0_alphaR_0_C_1_Q_0_L_0_A_0_regfunC_square_Q_abs_cut20_L_square_actC_square_Q_no_tag__seed_0'\n",
    "all_log, modellist, configlist = log_tflogs_from_path(tfboardroot)\n",
    "\n",
    "print(modellist)\n",
    "print(configlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_metrics = metriclist\n",
    "# plot_metrics = ['reward','avoid_rep']\n",
    "\n",
    "num_models = human_stats.shape[0]\n",
    "print('num_models: ', num_models)\n",
    "\n",
    "all_plotlog = None\n",
    "\n",
    "for metric in plot_metrics:\n",
    "    sub_human_stats = human_stats[['Model', metric+'-mean', metric+'-std']]\n",
    "    sub_log = all_log.loc[all_log['config']==default_config]\n",
    "    sub_log = sub_log.loc[sub_log['Metric']=='est_reward_dual_{}_normalized'.format(metric)]\n",
    "    sub_human_stats = sub_human_stats.sort_values(by=[metric+'-mean', metric+'-std'])\n",
    "    sub_human_stats = sub_human_stats.iloc[[0,6,12,23]]\n",
    "    \n",
    "    selected_models = list(sub_human_stats['Model'])\n",
    "    \n",
    "    sub_log.index = sub_log['Model']\n",
    "    sub_log = sub_log.loc[selected_models]\n",
    "    \n",
    "    # change model name\n",
    "    model_dict = {\n",
    "        selected_models[0]: 'Model 0%',\n",
    "        selected_models[1]: 'Model 25%',\n",
    "        selected_models[2]: 'Model 50%',\n",
    "        selected_models[3]: 'Model 100%',\n",
    "    }\n",
    "    sub_log['Model'] = sub_log['Model'].apply(lambda x: model_dict[x])\n",
    "    sub_human_stats['Model'] = sub_human_stats['Model'].apply(lambda x: model_dict[x])\n",
    "    plot_log = get_plot_df(sub_human_stats, sub_log, metric)\n",
    "    plot_log['Metric'] = metric\n",
    "    if all_plotlog is None:\n",
    "        all_plotlog = plot_log\n",
    "    else:\n",
    "        all_plotlog = all_plotlog.append(plot_log)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.FacetGrid(all_plotlog, col='Metric', col_wrap=3,\n",
    "                  height=3, aspect=1.5,sharey=False)\n",
    "def mylineplot(x, y, h, s, **kwargs):\n",
    "    sns.lineplot(x=x, y=y, hue=h, style=s, **kwargs)\n",
    "g.map(mylineplot, \n",
    "      'Steps', 'Values','Model','Evaluation',\n",
    "       ci='sd')\n",
    "g.add_legend();\n",
    "g.savefig(plotroot+'learning_curve.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Human vs. OPE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_plotlog = gather_ope_logs_by_config(all_log, metriclist, default_config)\n",
    "bleu_ppl_stats = human_stats[['Model', 'BLEU', 'ppl']].set_index('Model')\n",
    "all_plotlog = all_plotlog.join(bleu_ppl_stats)\n",
    "all_plotlog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ope vs human\n",
    "g = sns.lmplot(x=\"Human-mean\", y='OPE', col='Metric', hue='Metric',\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'ope_vs_human.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for m in all_plotlog.Metric.unique():\n",
    "    tempdf = all_plotlog[all_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['OPE']\n",
    "    X = X[~y.isna()]\n",
    "    y = y[~y.isna()]\n",
    "\n",
    "#     model = sm.OLS(y, X)\n",
    "#     results = model.fit()\n",
    "#     print('{} R2: {:.4f}'.format(m, results.rsquared))\n",
    "    print('{} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "#     print(results.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \n",
    "from scipy import stats\n",
    "def r2(x, y):\n",
    "    return stats.pearsonr(x, y)[0] ** 2\n",
    "g = sns.jointplot('Human-mean', 'OPE', kind=\"reg\", data=all_plotlog, stat_func=r2)\n",
    "g.savefig(plotroot+'ope_vs_human_all.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## vs. BLEU PPL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.lmplot(x=\"Human-mean\", y='BLEU', col='Metric', hue='Metric',\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'bleu_vs_human.pdf')\n",
    "for m in all_plotlog.Metric.unique():\n",
    "    tempdf = all_plotlog[all_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['BLEU']\n",
    "    print('BLEU {} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "g = sns.lmplot(x=\"Human-mean\", y='ppl', col='Metric', hue='Metric',\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'ppl_vs_human.pdf')\n",
    "for m in all_plotlog.Metric.unique():\n",
    "    tempdf = all_plotlog[all_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['ppl']\n",
    "    print('ppl {} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "# g = sns.jointplot(metric+\"-mean\", \"ppl\", kind=\"reg\", data=all_plotlog, stat_func=r2)\n",
    "# g.savefig(plotroot+'ppl_vs_human.pdf')\n",
    "# g = sns.jointplot(metric+\"-mean\", 'OPE', kind=\"reg\", data=all_plotlog, stat_func=r2)\n",
    "# g.savefig(plotroot+'ope_vs_human.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Error Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.lmplot(x=\"Human-mean\", y='Error', col='Metric', hue='Metric', order=2,\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.set(ylim=(-0.008, None))\n",
    "for ax, (_, subdata) in zip(g.axes, all_plotlog.groupby('Metric')):\n",
    "    ax2=ax.twinx()\n",
    "    sns.distplot(subdata[\"Human-mean\"], ax=ax2,color='#95a5a6')\n",
    "    plt.setp(ax2.get_yticklabels(), visible=False)\n",
    "    plt.setp(ax2.get_yticklines(), visible=False)\n",
    "    ax.patch.set_visible(True)\n",
    "    \n",
    "g.savefig(plotroot+'error_analysis.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AirDialogue Model-Human"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotroot=\"outputs/plotdir/air_ope/human/\"\n",
    "os.makedirs(plotroot,exist_ok=True)\n",
    "\n",
    "# load stats\n",
    "statspath = 'data/human_opedata/orig/stats.csv'\n",
    "\n",
    "human_stats = pd.read_csv(statspath)\n",
    "human_stats = human_stats.rename(columns={'model_name': 'Model'}).set_index('Model')\n",
    "\n",
    "# load SP stats\n",
    "statspath = 'data/selfplay_opedata/orig/stats.csv'\n",
    "human_stats_sp = pd.read_csv(statspath).add_prefix('SP-')\n",
    "human_stats_sp = human_stats_sp.rename(columns={'SP-model_name': 'Model'}).set_index('Model')\n",
    "human_stats = pd.concat([human_stats, human_stats_sp], axis=1, join='inner')\n",
    "\n",
    "human_stats_auto = {\n",
    "    'Model': ['5K','10K','20K','30K','40K','50K','75K','100K','150K','200K','250K','full',\n",
    "              '5K_w','10K_w','20K_w','30K_w','40K_w','50K_w','75K_w','100K_w','150K_w','200K_w','250K_w','full_w'], \n",
    "    'ppl': [2.671, 2.368, 2.141, 2.053, 2.121, 1.919, 1.843, 2.084, 2.021, 2.076, 1.949, 1.954,\n",
    "            2.673, 2.369, 2.141, 2.053, 2.121, 1.919, 1.843, 2.084, 2.021, 2.076, 1.949, 1.954], \n",
    "    'BLEU': [12.53, 19.07, 23.72, 24.49, 19.64, 29.98, 31.70, 20.10, 21.95, 20.24, 26.04, 25.41,\n",
    "             12.53, 19.07, 23.71, 24.49, 19.64, 29.98, 31.70, 20.11, 21.95, 20.23, 26.04, 25.41], \n",
    "}\n",
    "human_stats_auto = pd.DataFrame(data=human_stats_auto).set_index('Model')\n",
    "human_stats = pd.concat([human_stats_auto, human_stats], axis=1, join='inner').reset_index()\n",
    "\n",
    "\n",
    "metriclist = ['flight_score', 'reward', 'status_score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "human_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfboardroot = \"outputs/human_air_ope_all\"\n",
    "default_config = 'roberta-base_fix_false_share_true_freeze_true_epoch_500_invsqrt_adam_lr_1.5e-4_C_1_Q_2_L_10x100_BERT_1_warmup_30_mom_0.5_MAXNORM_1_WD_1e-4_BS_20x1_Linit_0_alphaR_0_C_1_Q_0_L_0_A_0_regfunC_square_Q_abs_cut20_L_square_actC_square_Q_no_tag__seed_0'\n",
    "all_log, modellist, configlist = log_tflogs_from_path(tfboardroot)\n",
    "\n",
    "print(modellist)\n",
    "print(configlist)\n",
    "all_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_metrics = metriclist\n",
    "# plot_metrics = ['reward','avoid_rep']\n",
    "\n",
    "num_models = human_stats.shape[0]\n",
    "print('num_models: ', num_models)\n",
    "\n",
    "all_plotlog = None\n",
    "\n",
    "for metric in plot_metrics:\n",
    "    sub_human_stats = human_stats[['Model', metric+'-mean', metric+'-std']]\n",
    "    sub_log = all_log.loc[all_log['config']==default_config]\n",
    "    sub_log = sub_log.loc[sub_log['Metric']=='est_reward_dual_{}_normalized'.format(metric)]\n",
    "    sub_human_stats = sub_human_stats.sort_values(by=[metric+'-mean', metric+'-std'])\n",
    "    sub_human_stats = sub_human_stats.iloc[[0,6,12,23]]\n",
    "    \n",
    "    selected_models = list(sub_human_stats['Model'])\n",
    "    \n",
    "    sub_log.index = sub_log['Model']\n",
    "    sub_log = sub_log.loc[selected_models]\n",
    "    \n",
    "    # change model name\n",
    "    model_dict = {\n",
    "        selected_models[0]: 'Model 0%',\n",
    "        selected_models[1]: 'Model 25%',\n",
    "        selected_models[2]: 'Model 50%',\n",
    "        selected_models[3]: 'Model 100%',\n",
    "    }\n",
    "    sub_log['Model'] = sub_log['Model'].apply(lambda x: model_dict[x])\n",
    "    sub_human_stats['Model'] = sub_human_stats['Model'].apply(lambda x: model_dict[x])\n",
    "    plot_log = get_plot_df(sub_human_stats, sub_log, metric)\n",
    "    plot_log['Metric'] = metric\n",
    "    if all_plotlog is None:\n",
    "        all_plotlog = plot_log\n",
    "    else:\n",
    "        all_plotlog = all_plotlog.append(plot_log)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.FacetGrid(all_plotlog, col='Metric', col_wrap=3,\n",
    "                  height=3, aspect=1.5,sharey=False)\n",
    "def mylineplot(x, y, h, s, **kwargs):\n",
    "    sns.lineplot(x=x, y=y, hue=h, style=s, **kwargs)\n",
    "g.map(mylineplot, \n",
    "      'Steps', 'Values','Model','Evaluation',\n",
    "       ci='sd')\n",
    "g.add_legend();\n",
    "g.savefig(plotroot+'learning_curve.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Human vs. OPE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_plotlog = gather_ope_logs_by_config(all_log, metriclist, default_config)\n",
    "bleu_ppl_stats = human_stats[['Model', 'BLEU', 'ppl']].set_index('Model')\n",
    "all_plotlog = all_plotlog.join(bleu_ppl_stats)\n",
    "all_splog = None\n",
    "for m in metriclist:\n",
    "    sp_log = human_stats[['Model', 'SP-'+m+'-mean', 'SP-'+m+'-std']].set_index('Model')\n",
    "    sp_log = sp_log.rename(columns = {'SP-'+m+'-mean': 'SP-mean', 'SP-'+m+'-std': 'SP-std'})\n",
    "    sp_log['Metric'] = m\n",
    "    if all_splog is None:\n",
    "        all_splog = sp_log\n",
    "    else:\n",
    "        all_splog = all_splog.append(sp_log)\n",
    "\n",
    "all_plotlog = all_plotlog.reset_index().set_index(['Model', 'Metric'])\n",
    "all_splog = all_splog.reset_index().set_index(['Model', 'Metric'])\n",
    "all_plotlog = all_plotlog.join(all_splog).reset_index()\n",
    "all_plotlog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ope vs human\n",
    "g = sns.lmplot(x=\"Human-mean\", y='OPE', col='Metric', hue='Metric',\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'ope_vs_human.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for m in all_plotlog.Metric.unique():\n",
    "    tempdf = all_plotlog[all_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['OPE']\n",
    "    X = X[~y.isna()]\n",
    "    y = y[~y.isna()]\n",
    "\n",
    "#     model = sm.OLS(y, X)\n",
    "#     results = model.fit()\n",
    "#     print('{} R2: {:.4f}'.format(m, results.rsquared))\n",
    "    print('{} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "#     print(results.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \n",
    "from scipy import stats\n",
    "def r2(x, y):\n",
    "    return stats.pearsonr(x, y)[0] ** 2\n",
    "g = sns.jointplot('Human-mean', 'OPE', kind=\"reg\", data=all_plotlog, stat_func=r2)\n",
    "g.savefig(plotroot+'ope_vs_human_all.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## vs. BLEU PPL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.lmplot(x=\"Human-mean\", y='BLEU', col='Metric', hue='Metric',\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'bleu_vs_human.pdf')\n",
    "for m in all_plotlog.Metric.unique():\n",
    "    tempdf = all_plotlog[all_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['BLEU']\n",
    "    print('BLEU {} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "g = sns.lmplot(x=\"Human-mean\", y='ppl', col='Metric', hue='Metric',\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'ppl_vs_human.pdf')\n",
    "for m in all_plotlog.Metric.unique():\n",
    "    tempdf = all_plotlog[all_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['ppl']\n",
    "    print('ppl {} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "g = sns.lmplot(x=\"Human-mean\", y='SP-mean', col='Metric', hue='Metric',\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'selfplay_vs_human.pdf')\n",
    "for m in all_plotlog.Metric.unique():\n",
    "    tempdf = all_plotlog[all_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['SP-mean']\n",
    "    print('Selfplay {} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "# g = sns.jointplot(metric+\"-mean\", \"ppl\", kind=\"reg\", data=all_plotlog, stat_func=r2)\n",
    "# g.savefig(plotroot+'ppl_vs_human.pdf')\n",
    "# g = sns.jointplot(metric+\"-mean\", 'OPE', kind=\"reg\", data=all_plotlog, stat_func=r2)\n",
    "# g.savefig(plotroot+'ope_vs_human.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## vs. Selfplay flight_score > 0.50, reward > 0.65, status_score > 0.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_plotlog = all_plotlog[(all_plotlog['Metric']=='flight_score') & (all_plotlog['Human-mean']>0.51)]\n",
    "sub_plotlog = sub_plotlog.append(all_plotlog[(all_plotlog['Metric']=='reward') & (all_plotlog['Human-mean']>0.65)])\n",
    "sub_plotlog = sub_plotlog.append(all_plotlog[(all_plotlog['Metric']=='status_score') & (all_plotlog['Human-mean']>0.7)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.lmplot(x=\"Human-mean\", y='BLEU', col='Metric', hue='Metric',\n",
    "               data=sub_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'bleu_vs_human_top.pdf')\n",
    "for m in sub_plotlog.Metric.unique():\n",
    "    tempdf = sub_plotlog[sub_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['BLEU']\n",
    "    print('BLEU {} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "g = sns.lmplot(x=\"Human-mean\", y='ppl', col='Metric', hue='Metric',\n",
    "               data=sub_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'ppl_vs_human_top.pdf')\n",
    "for m in sub_plotlog.Metric.unique():\n",
    "    tempdf = sub_plotlog[sub_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['ppl']\n",
    "    print('ppl {} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "g = sns.lmplot(x=\"Human-mean\", y='OPE', col='Metric', hue='Metric',\n",
    "               data=sub_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'ope_vs_human_top.pdf')\n",
    "for m in sub_plotlog.Metric.unique():\n",
    "    tempdf = sub_plotlog[sub_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['OPE']\n",
    "    print('OPE {} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "g = sns.lmplot(x=\"Human-mean\", y='SP-mean', col='Metric', hue='Metric',\n",
    "               data=sub_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'selfplay_vs_human_top.pdf')\n",
    "for m in sub_plotlog.Metric.unique():\n",
    "    tempdf = sub_plotlog[sub_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['SP-mean']\n",
    "    print('Selfplay {} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "# g = sns.jointplot(metric+\"-mean\", \"ppl\", kind=\"reg\", data=sub_plotlog, stat_func=r2)\n",
    "# g.savefig(plotroot+'ppl_vs_human.pdf')\n",
    "# g = sns.jointplot(metric+\"-mean\", 'OPE', kind=\"reg\", data=sub_plotlog, stat_func=r2)\n",
    "# g.savefig(plotroot+'ope_vs_human.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Error Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sp_all_plotlog = all_plotlog.copy()\n",
    "# sp_all_plotlog['Error'] = (sp_all_plotlog['SP-mean'] - sp_all_plotlog['Human-mean'])\n",
    "# sp_all_plotlog['Evaluation'] = 'Selfplay'\n",
    "# ope_all_plotlog = all_plotlog.copy()\n",
    "# ope_all_plotlog['Evaluation'] = 'OPE'\n",
    "# error_all_plotlog = ope_all_plotlog.append(sp_all_plotlog)\n",
    "g = sns.lmplot(x=\"Human-mean\", y='Error', col='Metric', hue='Metric', order=2,\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.set(ylim=(-0.008, None))\n",
    "for ax, (_, subdata) in zip(g.axes, all_plotlog.groupby('Metric')):\n",
    "    ax2=ax.twinx()\n",
    "    sns.distplot(subdata[\"Human-mean\"], ax=ax2,color='#95a5a6')\n",
    "    plt.setp(ax2.get_yticklabels(), visible=False)\n",
    "    plt.setp(ax2.get_yticklines(), visible=False)\n",
    "    ax.patch.set_visible(True)\n",
    "    \n",
    "g.savefig(plotroot+'error_analysis.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Covai2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# plotroot=\"outputs/plotdir/convai2/hard/\"\n",
    "# tfboardroot = 'outputs/convai2_ope_all_hard'\n",
    "\n",
    "plotroot=\"outputs/plotdir/convai2/all/\"\n",
    "tfboardroot = 'outputs/convai2_ope_all'\n",
    "default_config = 'roberta-base_fix_false_share_true_freeze_true_epoch_300_invsqrt_adam_lr_1e-4_C_1_Q_2_L_10x100_BERT_1_warmup_30_mom_0.5_MAXNORM_1_WD_1e-4_BS_20x1_Linit_-0.01_alphaR_0_C_1_Q_0_L_0_A_0_regfunC_square_Q_abs_cut20_L_square_actC_square_Q_no_tag__seed_0'\n",
    "\n",
    "os.makedirs(plotroot,exist_ok=True)\n",
    "\n",
    "# load stats\n",
    "statspath = 'data/convai2/orig/stats.csv'\n",
    "\n",
    "human_stats = pd.read_csv(statspath)\n",
    "human_stats = human_stats.rename(columns={'model_name': 'Model'})\n",
    "human_stats = human_stats.loc[human_stats['Model'] != 'human_eval']\n",
    "metriclist = [i.replace('-mean','') for i in list(human_stats.columns) if i.endswith('-mean')]\n",
    "print(metriclist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "human_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_log, modellist, configlist = log_tflogs_from_path(tfboardroot)\n",
    "\n",
    "print(modellist)\n",
    "print(configlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot Training Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_metrics = metriclist\n",
    "# plot_metrics = ['reward','avoid_rep']\n",
    "\n",
    "num_models = human_stats.shape[0]\n",
    "print('num_models: ', num_models)\n",
    "\n",
    "all_plotlog = None\n",
    "\n",
    "for metric in plot_metrics:\n",
    "    sub_human_stats = human_stats[['Model', metric+'-mean', metric+'-std']]\n",
    "    sub_log = all_log.loc[all_log['config']==default_config]\n",
    "    sub_log = sub_log.loc[sub_log['Metric']=='est_reward_dual_{}_normalized'.format(metric)]\n",
    "    sub_human_stats = sub_human_stats.sort_values(by=[metric+'-mean', metric+'-std'])\n",
    "    sub_human_stats = sub_human_stats.iloc[[0,6,13,27]]\n",
    "    \n",
    "    selected_models = list(sub_human_stats['Model'])\n",
    "    \n",
    "    sub_log.index = sub_log['Model']\n",
    "    sub_log = sub_log.loc[selected_models]\n",
    "    \n",
    "    # change model name\n",
    "    model_dict = {\n",
    "        selected_models[0]: 'Model 0%',\n",
    "        selected_models[1]: 'Model 25%',\n",
    "        selected_models[2]: 'Model 50%',\n",
    "        selected_models[3]: 'Model 100%',\n",
    "    }\n",
    "    sub_log['Model'] = sub_log['Model'].apply(lambda x: model_dict[x])\n",
    "    sub_human_stats['Model'] = sub_human_stats['Model'].apply(lambda x: model_dict[x])\n",
    "    plot_log = get_plot_df(sub_human_stats, sub_log, metric)\n",
    "    plot_log['Metric'] = metric\n",
    "    if all_plotlog is None:\n",
    "        all_plotlog = plot_log\n",
    "    else:\n",
    "        all_plotlog = all_plotlog.append(plot_log)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.FacetGrid(all_plotlog, col='Metric', col_wrap=5,\n",
    "                  height=3, aspect=1.5,sharey=False)\n",
    "def mylineplot(x, y, h, s, **kwargs):\n",
    "    sns.lineplot(x=x, y=y, hue=h, style=s, **kwargs)\n",
    "g.map(mylineplot, \n",
    "      'Steps', 'Values','Model','Evaluation',\n",
    "       ci='sd')\n",
    "g.add_legend();\n",
    "g.savefig(plotroot+'learning_curve.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Human vs OPE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_plotlog = gather_ope_logs_by_config(all_log, metriclist, default_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ope vs human\n",
    "g = sns.lmplot(x=\"Human-mean\", y='OPE', col='Metric', hue='Metric',\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.savefig(plotroot+'ope_vs_human.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for m in all_plotlog.Metric.unique():\n",
    "    tempdf = all_plotlog[all_plotlog.Metric == m]\n",
    "    X = tempdf['Human-mean']\n",
    "    y = tempdf['OPE']\n",
    "\n",
    "#     model = sm.OLS(y, X)\n",
    "#     results = model.fit()\n",
    "#     print('{} R2: {:.4f}'.format(m, results.rsquared))\n",
    "    print('{} R2: {:.4f}'.format(m, stats.pearsonr(X, y)[0] ))\n",
    "#     print(results.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \n",
    "from scipy import stats\n",
    "def r2(x, y):\n",
    "    return stats.pearsonr(x, y)[0] ** 2\n",
    "g = sns.jointplot('Human-mean', 'OPE', kind=\"reg\", data=all_plotlog, stat_func=r2)\n",
    "g.savefig(plotroot+'ope_vs_human_all.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Error Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# g = sns.FacetGrid(all_plotlog, col='Metric', col_wrap=5,\n",
    "#                   height=3, aspect=1,sharey=False,sharex=False)\n",
    "# g.map(sns.jointplot, \n",
    "#       'Human-mean', 'OPE', kind=\"reg\")\n",
    "g = sns.lmplot(x=\"Human-mean\", y='Error', col='Metric', hue='Metric', order=2,\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.set(ylim=(-0.008, None))\n",
    "for ax, (_, subdata) in zip(g.axes, all_plotlog.groupby('Metric')):\n",
    "    ax2=ax.twinx()\n",
    "    sns.distplot(subdata[\"Human-mean\"], ax=ax2,color='#95a5a6')\n",
    "    plt.setp(ax2.get_yticklabels(), visible=False)\n",
    "    plt.setp(ax2.get_yticklines(), visible=False)\n",
    "    ax.patch.set_visible(True)\n",
    "    \n",
    "g.savefig(plotroot+'error_analysis.pdf')\n",
    "                            \n",
    "#g = sns.FacetGrid(all_plotlog, hue='Metric', col='Metric', col_wrap=5, ).map(sns.distplot, \"Human-mean\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## vs. Auxilary Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_plotlog = gather_ope_logs_by_config(all_log, metriclist, default_config)\n",
    "default_plotlog['config'] = 'default'\n",
    "aux_plotlog = gather_ope_logs_by_config(all_log, metriclist, config = 'roberta-base_fix_false_share_true_freeze_true_epoch_300_invsqrt_adam_lr_1e-4_C_1_Q_2_L_10x100_BERT_1_warmup_30_mom_0.5_MAXNORM_1_WD_1e-4_BS_20x1_Linit_-0.01_alphaR_0_C_1_Q_0_L_0_A_1_regfunC_square_Q_abs_cut20_L_square_actC_square_Q_no_tag__seed_0')\n",
    "aux_plotlog['config'] = 'with Aux. Loss'\n",
    "all_plotlog = default_plotlog.append(aux_plotlog)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.lmplot(x=\"Human-mean\", y='Error', col='Metric', hue=\"config\", order=2,\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.set(ylim=(-0.008, None))\n",
    "g.savefig(plotroot+'error_vs_auxloss.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Auxilary loss is useless"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## vs Data Size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_log_half, _, _ = log_tflogs_from_path('outputs/convai2_ope_all_half/')\n",
    "config_half = \"roberta-base_fix_false_share_true_freeze_true_epoch_300_invsqrt_adam_lr_1e-4_C_1_Q_2_L_10x100_BERT_1_warmup_30_mom_0.5_MAXNORM_1_WD_1e-4_BS_20x1_Linit_-0.01_alphaR_0_C_1_Q_0_L_0_A_0_regfunC_square_Q_abs_cut20_L_square_actC_square_Q_no_tag__seed_0\"\n",
    "plotlog_half = gather_ope_logs_by_config(all_log_half, metriclist, config_half, last_n=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ope vs human half\n",
    "g = sns.lmplot(x=\"Human-mean\", y='OPE', col='Metric', hue='Metric',\n",
    "               data=plotlog_half, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False,robust=True)\n",
    "g.savefig(plotroot+'ope_vs_human_half.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for m in plotlog_half.Metric.unique():\n",
    "    tempdf = plotlog_half[plotlog_half.Metric == m]\n",
    "    tempdf = tempdf[~tempdf.OPE.isnull()]\n",
    "    X = np.array(tempdf['Human-mean'])\n",
    "    y = np.array(tempdf['OPE'])\n",
    "\n",
    "    print('{} R2: {:.4f}'.format(m, stats.pearsonr(y, X)[0] ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_log_small, _, _ = log_tflogs_from_path('outputs/convai2_ope_all_small/')\n",
    "config_small = \"roberta-base_fix_false_share_true_freeze_true_epoch_1000_invsqrt_adam_lr_1e-4_C_1_Q_2_L_10x100_BERT_1_warmup_30_mom_0.5_MAXNORM_1_WD_1e-4_BS_20x1_Linit_-0.01_alphaR_0_C_1_Q_0_L_0_A_0_regfunC_square_Q_abs_cut20_L_square_actC_square_Q_no_tag__seed_0\"\n",
    "plotlog_small = gather_ope_logs_by_config(all_log_small, metriclist, config_small, last_n=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ope vs human small\n",
    "g = sns.lmplot(x=\"Human-mean\", y='OPE', col='Metric', hue='Metric',\n",
    "               data=plotlog_small, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False,robust=True)\n",
    "g.savefig(plotroot+'ope_vs_human_small.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for m in plotlog_small.Metric.unique():\n",
    "    tempdf = plotlog_small[plotlog_half.Metric == m]\n",
    "    tempdf = tempdf[~tempdf.OPE.isnull()]\n",
    "    X = np.array(tempdf['Human-mean'])\n",
    "    y = np.array(tempdf['OPE'])\n",
    "\n",
    "    print('{} R2: {:.4f}'.format(m, stats.pearsonr(y, X)[0] ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_plotlog = gather_ope_logs_by_config(all_log, metriclist, default_config)\n",
    "all_plotlog['Data'] = '100%'\n",
    "plotlog_50 = plotlog_half.copy()\n",
    "plotlog_50['Data'] = '50%'\n",
    "all_plotlog = all_plotlog.append(plotlog_50)\n",
    "plotlog_10 = plotlog_small.copy()\n",
    "plotlog_10['Data'] = '10%'\n",
    "all_plotlog = all_plotlog.append(plotlog_10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.lmplot(x=\"Human-mean\", y='Error', col='Metric', hue='Data', order=2,\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.set(ylim=(-0.008, None))\n",
    "g.savefig(plotroot+'error_vs_datasize.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hard Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotroot=\"outputs/plotdir/convai2/hard/\"\n",
    "all_log_hard, _, _ = log_tflogs_from_path('outputs/convai2_ope_all_hard/')\n",
    "os.makedirs(plotroot,exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotlog_hard = gather_ope_logs_by_config(all_log_hard, metriclist, default_config, last_n=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ope vs human\n",
    "g = sns.lmplot(x=\"Human-mean\", y='OPE', col='Metric', hue='Metric',\n",
    "               data=plotlog_hard, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False,robust=True)\n",
    "g.savefig(plotroot+'ope_vs_human.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for m in plotlog_hard.Metric.unique():\n",
    "    tempdf = plotlog_hard[plotlog_hard.Metric == m]\n",
    "    tempdf = tempdf[~tempdf.OPE.isnull()]\n",
    "    X = np.array(tempdf['Human-mean'])\n",
    "    y = np.array(tempdf['OPE'])\n",
    "\n",
    "#     model = sm.OLS(y, X)\n",
    "#     results = model.fit()\n",
    "#     print('Parameters: ', results.params)\n",
    "#     print('{} R2: {:.4f}'.format(m, results.rsquared))\n",
    "#     print(results.summary())\n",
    "\n",
    "    print('{} R2: {:.4f}'.format(m, stats.pearsonr(y, X)[0] ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def r2(x, y):\n",
    "    return stats.pearsonr(x, y)[0] ** 2\n",
    "g = sns.jointplot('Human-mean', 'OPE', kind=\"reg\", data=plotlog_hard, stat_func=r2)\n",
    "g.savefig(plotroot+'ope_vs_human_all.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.lmplot(x=\"Human-mean\", y='Error', col='Metric', hue='Metric', order=2,\n",
    "               data=plotlog_hard, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.set(ylim=(-0.008, None))\n",
    "for ax, (_, subdata) in zip(g.axes, plotlog_hard.groupby('Metric')):\n",
    "    ax2=ax.twinx()\n",
    "    sns.distplot(subdata[\"Human-mean\"], ax=ax2,color='#95a5a6')\n",
    "    plt.setp(ax2.get_yticklabels(), visible=False)\n",
    "    plt.setp(ax2.get_yticklines(), visible=False)\n",
    "    ax.patch.set_visible(True)\n",
    "    \n",
    "g.savefig(plotroot+'error_analysis.pdf')\n",
    "                            \n",
    "#g = sns.FacetGrid(all_plotlog, hue='Metric', col='Metric', col_wrap=5, ).map(sns.distplot, \"Human-mean\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_plotlog = gather_ope_logs_by_config(all_log, metriclist, default_config)\n",
    "default_plotlog['Data'] = 'Normal'\n",
    "plotlog_hard = gather_ope_logs_by_config(all_log_hard, metriclist, default_config)\n",
    "plotlog_hard['Data'] = 'Hard'\n",
    "all_plotlog = default_plotlog.append(plotlog_hard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.lmplot(x=\"Human-mean\", y='Error', col='Metric', hue='Data', order=2,\n",
    "               data=all_plotlog, col_wrap=5, height=3,sharey=False,sharex=False,truncate=False)\n",
    "g.set(ylim=(-0.008, None))\n",
    "for ax, (_, subdata) in zip(g.axes, all_plotlog.groupby('Metric')):\n",
    "    ax2=ax.twinx()\n",
    "    sns.distplot(subdata[\"Human-mean\"], ax=ax2,color='#95a5a6')\n",
    "    plt.setp(ax2.get_yticklabels(), visible=False)\n",
    "    plt.setp(ax2.get_yticklines(), visible=False)\n",
    "    ax.patch.set_visible(True)\n",
    "g.savefig(plotroot+'error_normal_vs_hard_data.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc-autonumbering": true,
  "toc-showcode": false,
  "toc-showmarkdowntxt": false,
  "toc-showtags": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
